# -*- coding: utf-8 -*-
import requests
from lxml import etree
import re
import time
from loguru import logger
import random
import datetime
import execjs
import pymysql
import os
import threading
import sys
from queue import Queue, Empty
import json
from fake_useragent import UserAgent
import concurrent.futures
import traceback
from src.service.requests_middler import requests_sys
from src.service.requests_middler import proxy_requests_sys
from src.service.requests_middler import insert_shipping_info
from src.service.requests_middler import read_excel
from src.service.requests_middler import read_transport_json
from src.service.requests_middler import get_proxies

ip_queue = Queue()

ua = UserAgent()
random_user_agent = ua.random

stop_threading_event = threading.Event()
current_working_directory = os.getcwd()

total_requests = 0
loss_requests = 0


def proxies_queue():
    # 获取代理并放入队列
    if ip_queue.empty():
        proxy = get_proxies()
        logger.info(f"获取ip代理成功: {proxy}")
        if proxy:
            ip_queue.put(proxy)
        else:
            logger.error(f"获取ip代理失败")
    else:
        proxy = ip_queue.get()
        return proxy


def get_MAEU():
    # file_path = "maeu_ports.json"
    # file_path = os.path.join(current_working_directory + r'/src/crawler', file_path)
    file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')
    if os.path.exists(file_path):
        with open(file_path, "r") as file:
            return json.load(file)
    url = "https://api.maersk.com.cn/synergy/schedules/active-ports"
    params = {
        "carrierCodes": "MAEU"
    }
    headers = {
        "Host": "api.maersk.com.cn",
        "pragma": "no-cache",
        "cache-control": "no-cache",
        "sec-ch-ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Microsoft Edge\";v=\"128\"",
        "accept": "application/json",
        "consumer-key": "uXe7bxTHLY0yY0e8jnS6kotShkLuAAqG",
        "sec-ch-ua-mobile": "?0",
        "user-agent": random_user_agent,
        "sec-ch-ua-platform": "\"Windows\"",
        "origin": "https://www.maersk.com.cn",
        "sec-fetch-site": "same-site",
        "sec-fetch-mode": "cors",
        "sec-fetch-dest": "empty",
        "referer": "https://www.maersk.com.cn/",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "priority": "u=1, i"
    }
    res = requests_sys(url, headers=headers, params=params, GET=True)
    if res:
        data = res.json()
        with open(file_path, 'w', encoding='utf-8') as json_file:
            json.dump(data, json_file, ensure_ascii=False, indent=4)
        return data
    elif res == 0:
        sys.exit()


def maersk_transport(origin_city, destination_city):
    global total_requests
    global loss_requests
    url = "https://api.maersk.com.cn/delivery-network/mepc/routing/maeu/future-schedules"
    date = datetime.datetime.now().strftime('%Y-%m-%d')
    port_mapping = {}
    # 获取港口信息
    Menu_json = get_MAEU()
    for info in Menu_json['ports']:
        port_mapping[info['portName'].upper()] = {
            'portCode': info.get('portCode'),
            'unLocationCode': info['unLocationCode']
        }
    if "TIANJIN" == origin_city.upper():
        origin_info = port_mapping.get("Xingang".upper())
    else:
        origin_info = port_mapping.get(origin_city.upper())
    destination_info = port_mapping.get(destination_city.upper())
    if origin_info and destination_info:
        total_requests += 1
        origin_code = origin_info['portCode']
        origin_unLocationCode = origin_info['unLocationCode']
        destination_code = destination_info['portCode']
        des_unLocationCode = destination_info['unLocationCode']
        if origin_code and destination_code:
            headers = {
                "Host": "api.maersk.com.cn",
                "pragma": "no-cache",
                "cache-control": "no-cache",
                "sec-ch-ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Microsoft Edge\";v=\"128\"",
                "accept": "application/json",
                "consumer-key": "uXe7bxTHLY0yY0e8jnS6kotShkLuAAqG",
                "sec-ch-ua-mobile": "?0",
                "user-agent": random_user_agent,
                "sec-ch-ua-platform": "\"Windows\"",
                "origin": "https://www.maersk.com.cn",
                "sec-fetch-site": "same-site",
                "sec-fetch-mode": "cors",
                "sec-fetch-dest": "empty",
                "referer": "https://www.maersk.com.cn/",
                "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
                "priority": "u=1, i"
            }
            params = {
                "from": origin_code,
                "to": destination_code,
                "containerIsoCode": "42G1",
                "fromServiceMode": "CY",
                "toServiceMode": "CY",
                "numberOfWeeks": "4",
                "dateType": "D",
                "date": f"{date}",
                "vesselFlag": "",
                "cargoType": "DRY",
                "containerType": "DRY",
                "containerLength": "40",
                "brandCode": "maeu"
            }
            retry_count = 0
            while retry_count < 3:
                proxies = proxies_queue()
                res = requests_sys(url, headers=headers, params=params, GET=True)
                # res = proxy_requests_sys(url, headers=headers, params=params, GET=True)
                if res:
                    ip_queue.put(proxies)
                    json_data = res.json()
                    logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city}")
                    logger.info(f"当前请求总数: {total_requests}, 失败请求总数: {loss_requests}")
                    if json_data.get("products"):
                        for info in json_data["products"]:
                            for info2 in info['schedules']:
                                item = {}
                                if "CNTXG" == origin_unLocationCode:
                                    origin_unLocationCode = "CNTJN"
                                item['origin_code'] = f"{origin_city.upper()}({origin_unLocationCode})"
                                item['destination_code'] = (f"{destination_city},"
                                                            f"{check_country_code(des_unLocationCode)}"
                                                            f"({des_unLocationCode})")  # check
                                item['depart_date'] = info2['fromLocation']['date']
                                item['arrive_date'] = info2['toLocation']['date']
                                item['transport_contractor'] = "Maersk"
                                item['shipping_lanes'] = info2["scheduleDetails"][0]["serviceName"]
                                departure_date = datetime.datetime.strptime(item['depart_date'], '%Y-%m-%d')
                                arrival_date = datetime.datetime.strptime(item['arrive_date'], '%Y-%m-%d')
                                days_difference = (arrival_date - departure_date).days
                                item['haulage_time'] = days_difference
                                item['ship_name'] = info2["scheduleDetails"][0]["transport"]["vessel"]["name"]
                                item['voyage_number'] = info2["scheduleDetails"][0]["transport"]["voyageNumber"]
                                item['transfer_code'] = None
                                transfer_code = ''
                                code = ''
                                if len(info2['scheduleDetails']) >= 2:
                                    if not info2['scheduleDetails'][-1]['transport']['vessel']:
                                        for transfer in list(reversed(info2['scheduleDetails']))[1:]:
                                            if transfer['transport']['vessel']:
                                                site_code = transfer['toLocation']['siteGeoId']
                                                link = f'https://api.maersk.com.cn/synergy/reference-data/geography/locations/{site_code}'
                                                resp = requests.get(link, headers=headers).json()
                                                site_name = resp['cityName']
                                                code = resp['unLocCode']
                                                transfer_code = f'{site_name}({code})'
                                                break
                                        if transfer_code != '' and check_code(code):
                                            item['transfer_code'] = transfer_code
<<<<<<< .mine
                                # insert_shipping_info(item)
                                print(item)
||||||| .r58155
                                insert_shipping_info(item)
                                # print(item)
=======
                                print(item)
                                insert_shipping_info(item)

>>>>>>> .r59648
                    else:
                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据")
                        read_transport_json(transport='Maersk', origin_city=origin_city, des_city=destination_city,
                                            fail_result="未查询到数据")
                        loss_requests += 1
                        retry_count += 1

                    break
                elif res == 0:
                    # read_transport_json(transport='Maersk', origin_city=origin_city, des_city=destination_city, fail_result=None)
                    logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据, 请求超时")
                    retry_count += 1

            if retry_count == 3:
                read_transport_json(transport='Maersk', origin_city=origin_city, des_city=destination_city,
                                    fail_result="请求超时")
        else:
            read_transport_json(transport='Maersk', origin_city=origin_city, des_city=destination_city,
                                fail_result="未有对应编号")
            logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据")
            loss_requests += 1


def check_code(code):
    file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')
    with open(file_path, 'r') as f:
        result = f.read()
        f.close()
    null = ''
    true = ''
    false = ''
    result = eval(result)['ports']
    for res in result:
        if code == res['unLocationCode'] and (code[0:2] == 'US' or code[0:2] == 'CA'):
            return True
    return False


def check_country_code(code):
    file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')
    with open(file_path, 'r') as f:
        result = f.read()
        f.close()
    null = ''
    true = ''
    false = ''
    result = eval(result)['ports']
    for res in result:
        if code == res['unLocationCode'] and (code[0:2] == 'US' or code[0:2] == 'CA'):
            try:
                code = res['regionCode']
                return code
            except:
                return ''
    return ''


# Maersk
def process_route(origin_city, destination_list):
    for destination_city in destination_list:
        maersk_transport(origin_city, destination_city)


def maersk_spider_run():
    global total_requests
    global loss_requests
    # origin_city_list = [
    #     "Shanghai", "XIAMEN", "NANSHA", "QINGDAO", "YANTIAN", "NINGBO"
    # ]
    #
    # destination_list = [
    #     "Los Angeles", "Long Beach", "Atlanta", "Baltimore", "Boston", "Buffalo", "Calgary", "Charleston",
    #     "Chicago", "Chippewa Falls", "Cincinnati", "Cleveland", "Dallas", "Denver",
    #     "Detroit", "Edmonton", "EL Paso", "Halifax", "Houston", "Jacksonville",
    #     "Kansas City", "Louisville", "Memphis", "Miami", "Minneapolis", "Montreal",
    #     "Nashville", "New Orleans", "New York", "Norfolk", "Oakland", "Omaha",
    #     "Philadelphia", "Pittsburgh", "Portland", "Prince Rupert", "Saint Louis",
    #     "Salt Lake City", "San Antonio", "Saskatoon", "Savannah", "Seattle",
    #     "Tacoma", "Tampa", "Toronto", "Vancouver", "Winnipeg"
    # ]
    json_path_one = r"origin_and_destination.json"
    # json_path_one = r"origin_and_destination - 副本.json"
    json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), json_path_one)
    with open(f'{json_path}', 'r', encoding='utf-8') as f:
        tmp = json.loads(f.read())

    origin_city_list = tmp['origin_city_list']
    destination_list = tmp['destination_list']
    del tmp, json_path

    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     futures = [executor.submit(process_route, origin_city, destination_list) for origin_city in origin_city_list]

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = []
        for origin_city in origin_city_list:
            future = executor.submit(process_route, origin_city, destination_list)
            futures.append(future)
<<<<<<< .mine
            time.sleep(5)
||||||| .r58155
            time.sleep(1)
=======
            time.sleep(2)
>>>>>>> .r59648

        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                # 输出异常详细信息，包括出错的行号
                print(f"An error occurred: {e}")
                print(traceback.format_exc())

    read_excel(transport="Maersk", total_requests=total_requests, loss_requests=loss_requests)


if __name__ == '__main__':
    maersk_spider_run()


